from gm.api import *
import datetime
import numpy as np
import pandas as pd
import statsmodels.api as sm





#print(行业股票_list)

"""
首先是去极值的处理，去极值的本质就是使用均值的几倍标准差上下限去取代超过阈值的那个值
edge_up = r.mean()+std*r.std()
edge_low = r.mean()-std*r.std()
r[r>edge_up] = edge_up
r[r<edge_low] = edge_low
"""#去极值的处理说明
#来源：https://www.joinquant.com/post/12445?tag=algorithm
"""
下面是开始对行业中性化的处理：
    具体方法：根据大部分的研报对于中性化的处理，主要的方法是利用回归得到一个与风险因子线性无关的因子，
    即通过建立线性回归，提取残差作为中性化后的新因子。这样处理后的中性化因子与风险因子之间的相关性严格为零。
"""


def detal_edge(real,std_num = 3):

    edge_up = real.mean() + std_num * real.std()
    edge_low = real.mean() - std_num * real.std()

    real[real > edge_up] = edge_up
    real[real < edge_low] = edge_low


    return real

def fast_detal_edge(real, std_num=3):

    _real = real
    real = []
    for _ in _real:
        real.append(_)

    edge_up = np.mean(real) + std_num * np.std(real)
    edge_low = np.mean(real) - std_num * np.std(real)

    result = []
    for _ in real:

        if _ > edge_up:
            result.append(edge_up)
        elif _ < edge_low:
            result.append(edge_low)
        else:result.append(_)
    return result



# #在symbol中进行中性化因子处理
# def get_neutralized_factor(symbol,factor,symbol_list,now):
#
#     last_day = get_previous_trading_date("SHSE",now)
#
#     _symbol_list = symbol_list
#     symbol_list = []
#     for _ in _symbol_list:
#         symbol_list.append(_)
#
#
#     #根据市值进行中性化处理
#     df = get_fundamentals(table='trading_derivative_indicator', symbols=symbol_list, start_date=last_day,
#                           end_date=last_day, fields="TOTMKTCAP",df=True)
#     df = df.dropna()
#     df = df.sort_values(["symbol"])
#
#
#     factor_with_table = {}
#     for line in open("../tools/单因子表列.txt", encoding='UTF-8'):
#         line = line.strip('\n')[0:]
#         lines = line.split(",")
#         factor_with_table[lines[1]] = lines[0]
#     table_name = factor_with_table[factor]
#
#
#
#     df[factor] = 0
#     for _ in range(len(df)):
#         _symbol = df.iloc[_,0]
#         #获取每股净资产
#         _df = get_fundamentals(table=table_name, symbols=_symbol, start_date=last_day,
#                           end_date=last_day, fields=factor,df=True)
#         try:
#             df.iloc[_, 4] = (_df[factor].values)
#         except:
#             df.iloc[_, 4] = 0
#
#     df = df[df[factor] > 0]
#
#
#     TOTMKTCAP = df["TOTMKTCAP"].values  #这个是x
#     NAPSNEWP = df[factor].values   #这个是Y
#
#
#
#     TOTMKTCAP = sm.add_constant(TOTMKTCAP)
#
#     model = sm.OLS(NAPSNEWP, TOTMKTCAP)
#     results = model.fit()
#
#     df = get_fundamentals(table='trading_derivative_indicator', symbols=symbol, start_date=last_day,
#                           end_date=last_day, fields="TOTMKTCAP",df=True)
#     _x = df["TOTMKTCAP"].values #symbol的市值
#
#
#     _df = get_fundamentals(table=table_name, symbols=symbol, start_date=last_day,
#                            end_date=last_day, fields=factor, df=True)
#     _Y =  _df[factor].values  #待求的因子值
#
#     d_value = _Y - ( results.params[1] * _x + results.params[0])
#     return round(d_value[0],2)
#
# #在symbol中进行中性化因子处理
# def batch_get_neutralized_factor(symbol_list,factor,now,more_is_better = True):
#     """
#
#     :param symbol_list:
#     :param factor:
#     :param now:
#     :param more_is_better: 这个指的是因子本身，而不是计算出的值
#     :return:
#     """
#     last_day = get_previous_trading_date("SHSE",now)
#
#     _symbol_list = symbol_list
#     symbol_list = []
#     for _ in _symbol_list:
#         symbol_list.append(_)
#
#
#     df = pd.DataFrame([])
#     #根据市值进行中性化处理
#     _df = get_fundamentals(table='trading_derivative_indicator', symbols=symbol_list, start_date=last_day,
#                           end_date=last_day, fields="TOTMKTCAP",df=True)
#
#     if len(_df) == len(symbol_list):
#         df = _df
#         del df["pub_date"]
#         del df["end_date"]
#
#     else:
#         df["TOTMKTCAP"] = 0
#         for _ in range(len(df)):
#             _symbol = df.iloc[_, 0]
#             # 获取每个symbol的对应factor值
#             _df = get_fundamentals(table="trading_derivative_indicator", symbols=_symbol, start_date=last_day,
#                                    end_date=last_day, fields="TOTMKTCAP")
#             try:
#                 df.iloc[_, 1] = (_df[0]["TOTMKTCAP"])
#             except:
#                 df.iloc[_, 1] = 1e+15
#
#     _symbol_list = df["symbol"].values
#     symbol_list = []
#     for _ in _symbol_list:
#         symbol_list.append(_)
#
#
#     factor_with_table = {}
#     for line in open("../tools/单因子表列.txt", encoding='UTF-8'):
#         line = line.strip('\n')[0:]
#         lines = line.split(",")
#         factor_with_table[lines[1]] = lines[0]
#     table_name = factor_with_table[factor]
#
#
#
#
#     _df = get_fundamentals(table=table_name, symbols=symbol_list, start_date=last_day,
#                           end_date=last_day, fields=factor,df=True)
#
#     if len(_df) == len(symbol_list):
#         df[factor] = _df[factor]
#         del df["pub_date"]
#         del df["end_date"]
#     else:
#         df[factor] = 0
#         for _ in range(len(df)):
#             _symbol = df.iloc[_,0]
#             #获取每个symbol的对应factor值
#             _df = get_fundamentals(table=table_name, symbols=_symbol, start_date=last_day,
#                               end_date=last_day, fields=factor)
#             try:
#                 df.iloc[_, 2] = (_df[0][factor])
#             except:
#                 if more_is_better == True:
#                     df.iloc[_, 2] = -999
#                 else:
#                     df.iloc[_, 2] = 999
#
#
#     df["TOTMKTCAP"] = detal_edge(df["TOTMKTCAP"])  #这里进行去极值处理
#     df[factor] = detal_edge(df[factor])  #这里进行取极值处理
#
#     TOTMKTCAP = df["TOTMKTCAP"].values  #这个是x
#     factor_value = df[factor].values   #这个是Y
#
#
#
#     TOTMKTCAP = sm.add_constant(TOTMKTCAP)
#
#     model = sm.OLS(factor_value, TOTMKTCAP)
#     results = model.fit()
#
#
#     d_value_dict = {}
#
#     for _ in range(len(df)):
#         d_value = df.iloc[_,2] - ( results.params[1] * df.iloc[_,1] + results.params[0])
#         d_value_dict[df.iloc[_,0]] = d_value
#
#
#     return d_value_dict



def fast_batch_get_neutralized_factor(symbol_list,factor,now,more_is_better):
    """
    :param symbol_list:
    :param factor:
    :param now:
    :param more_is_better: 这个指的是因子本身，而不是计算出的值
    :return:
    """
    last_day = get_previous_trading_date("SHSE",now)
    _symbol_list = symbol_list
    symbol_list = []
    for _ in _symbol_list:
        symbol_list.append(_)
    TOTMKTCAP = []
    #根据市值进行中性化处理
    data = get_fundamentals(table='trading_derivative_indicator', symbols=symbol_list, start_date=last_day,
                          end_date=last_day, fields="TOTMKTCAP")

    if len(data) == len(symbol_list):
        for _ in data:
            try:
                TOTMKTCAP.append(_["TOTMKTCAP"])
            except:TOTMKTCAP.append(1e+15)
    else:
        for symbol in symbol_list:
            try:
                _data = get_fundamentals(table='trading_derivative_indicator', symbols=symbol, start_date=last_day,
                                        end_date=last_day, fields="TOTMKTCAP")

                TOTMKTCAP.append(_data[0]["TOTMKTCAP"])
            except:
                TOTMKTCAP.append(1e+15)

    factor_with_table = {}
    for line in open("../tools/单因子表列.txt", encoding='UTF-8'):
        line = line.strip('\n')[0:]
        lines = line.split(",")
        factor_with_table[lines[1]] = lines[0]
    table_name = factor_with_table[factor]


    factor_value_list = []
    data = get_fundamentals(table=table_name, symbols=symbol_list, start_date=last_day,
                          end_date=last_day, fields=factor)

    if len(data) == len(symbol_list):
        for _ in data:
            try:
                factor_value_list.append(_[factor])
            except:

                if more_is_better == True:
                    factor_value_list.append(-999)
                else:
                    factor_value_list.append(999)
    else:
        for symbol in symbol_list:
            try:
                _data = get_fundamentals(table=table_name, symbols=symbol, start_date=last_day,
                          end_date=last_day, fields=factor)
                factor_value_list.append(_data[0][factor])
            except:

                if more_is_better == True:
                    factor_value_list.append(-999)
                else:
                    factor_value_list.append(999)

    TOTMKTCAP = fast_detal_edge(TOTMKTCAP)   #这个是x
    factor_value_list = fast_detal_edge(factor_value_list)   #这个是Y

    _TOTMKTCAP = sm.add_constant(TOTMKTCAP)
    model = sm.OLS(factor_value_list, _TOTMKTCAP)
    results = model.fit()

    d_value_dict = {}

    for i in range(len(symbol_list)):
        d_value = factor_value_list[i] - ( results.params[1] * TOTMKTCAP[i] + results.params[0])
        d_value_dict[symbol_list[i]] = d_value

    return d_value_dict



if __name__ == "__main__":
    pass
    # set_token("a71a8083b68e73817e93f7f196b030482abe5939")
    # day_time, hour_and_mins = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')).split(" ")
    #
    # from MSCI_tools import msci_tools
    # MSCI_list = msci_tools.read_MSCI()
    # #MSCI_list = ["SZSE.300296","SHSE.600887"]
    # print("-----------------------------------------------------")
    # for _ in range(1):
    #     fast_result = fast_batch_get_neutralized_factor(MSCI_list,"PETTM",day_time,more_is_better=True)
    #     print(fast_result)

